# -*- coding: utf-8 -*-
import time
import json
import re

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from selenium.common.exceptions import TimeoutException
from lxml import etree
import pandas as pd

import os

# 专业伪装浏览器
def gen_browser(driver_path):#driver_path
    '''实例化一个driver'''
    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    options.add_argument('--disable-gpu')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-gpu')
    options.add_argument('--ignore-certificate-errors')
    options.add_argument('disable-infobars')
    options.add_argument("--disable-plugins-discovery") 
    # User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299
    # user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36"
    user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299"
    options.add_argument('user-agent="{0}"'.format(user_agent))
    # ############### 专业造假 ***************************
    def send(driver, cmd, params={}):
        '''
        向调试工具发送指令
        from: https://stackoverflow.com/questions/47297877/to-set-mutationobserver-how-to-inject-javascript-before-page-loading-using-sele/47298910#47298910
        '''
        resource = "/session/%s/chromium/send_command_and_get_result" % driver.session_id
        url = driver.command_executor._url + resource
        body = json.dumps({'cmd': cmd, 'params': params})
        response = driver.command_executor._request('POST', url, body)
        if response['status']:
            raise Exception(response.get('value'))
        return response.get('value')
    def add_script(driver, script):
        '''在页面加载前执行js'''
        send(driver, "Page.addScriptToEvaluateOnNewDocument", {"source": script})
    # 给 webdriver.Chrome 添加一个名为 add_script 的方法
    webdriver.Chrome.add_script = add_script # 这里（webdriver.Chrome）可能需要改，当调用不同的驱动时
    # *************** 专业造假 ###################
    browser = webdriver.Chrome(
        executable_path=driver_path,
        options=options
    )
      
    
    # ################## 辅助调试 *********************
    existed = {
        'executor_url': browser.command_executor._url,  # 浏览器可被远程连接调用的地址
        'session_id': browser.session_id  # 浏览器会话ID
    }
    # pprint(existed)
    # ********************* 辅助调试 ##################
    # ############### 专业造假 ***************************
    browser.add_script("""
    Object.defineProperty(navigator, 'webdriver', {
        get: () => false,
    });
    window.navigator.chrome = {
        runtime: {},
    };
    Object.defineProperty(navigator, 'languages', {
        get: () => ['zh-CN', 'zh']
    });
    Object.defineProperty(navigator, 'plugins', {
        get: () => [0, 1, 2],
    });
    """)
    # *************** 专业造假 ###################
    return browser


browser = gen_browser(driver_path="C:\\Users\\Administrator\\Downloads\\chromedriver_win32\\chromedriver.exe")
url = 'https://search.suning.com/笔记本/'

def write_to_csv(data):
    file_save_path = os.path.join(os.getcwd(),'suning.csv')
    # pass

    ls = []
    ls.append(data)
    df = pd.DataFrame(data=ls, index=None)
    if not os.path.exists(file_save_path):
        df.to_csv(file_save_path, encoding="UTF-8", index=False, mode='a', header=["评论量","商品描述", "商品封面", "商品单价"])
    else:
        df.to_csv(file_save_path, encoding="UTF-8", index=False, mode='a', header=False)

def parse_one_page(selector):
    try:
        products = selector.xpath('//div[@id="product-list"]/ul/li')
        # data = []
        for product in products:
            try:

                overimg = product.xpath(".//div/div/div[@class='res-img']/div/a/img/@src")[0].strip()
                # print("封面图：",overimg)
                desc = product.xpath(".//div/div/div[@class='res-info']/div[@class='title-selling-point']/a")[0].xpath('string(.)').replace('\n','')
                # print("描述：",desc)
                price = product.xpath(".//div/div/div[@class='res-info']/div[@class='price-box']/span")[0].xpath('string(.)').strip()
                # print("价格：",price)

                comment_quality = product.xpath(".//div/div/div[contains(@class,'evaluate-old')]/div[@class='info-evaluate']/a/i/text()")[0]
                # print("评价量：",comment)
              
                data_dict = {
                    'overimg':overimg,
                    'desc':desc,
                    'price':price,
                    'comment_quality': comment_quality
                }
                write_to_csv(data_dict)

            except Exception as e:
                # pass
                print(e)
    except Exception as e:
                # pass
            print(e)




def get_page():

    browser.get(url)
    wait = WebDriverWait(browser, 5)
    html = browser.page_source
    selector = etree.HTML(html)

    # 获取总页数
    page_more = selector.xpath('//span[@class="page-more"]/text()')
    total_page = int(page_more[0][1:-2])

    for page in range(1,total_page+1):
        
        el_ipt = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR,'#bottomPage'))
            )
        el_ipt.clear()
        el_ipt.send_keys(page)

        #  无头浏览器模式下 不能用这种方法去点击
        # el_submit = wait.until(
        #     EC.element_to_be_clickable((By.CSS_SELECTOR,'a.page-more.ensure'))
        #     )
        # el_submit.click()

        #  无头浏览器模式下 用这种方法去点击
        browser.execute_script("$('a.page-more.ensure').click()")

        print(page)
        time.sleep(1)

        # 一直滚动到最底部
        flag = True
        js1 = 'return document.body.scrollHeight'
        js2 = 'window.scrollTo(0, document.body.scrollHeight)'
        old_scroll_height = 0
        while flag:
            old_scroll_height = browser.execute_script(js1)
            browser.execute_script(js2) 
            if  browser.execute_script(js1) == old_scroll_height:
                flag = False
            time.sleep(1)
        time.sleep(3)
      
        html = browser.page_source
        selector = etree.HTML(html)
        parse_one_page(selector)
    browser.quit()




if __name__ == "__main__":
    
    get_page()

